Virginia Clean Energy Project

Extract, transform and load datasets from U.S. Energy Information Administration

This notebook includes code for retrieving and parsing datasetsfrom the bulk download facility maintained by the U.S. Energy Information Administration. Datasets currently retrieved include:

if(!("jsonlite" %in% installed.packages())) install.packages("jsonlite")
library(here)
library(readr)
library(tidyverse)
library(stringr)

t(manifest$dataset[[2]]) 

                   1                                                                                                                                                                                                                                           
last_updated       "2016-01-04T10:35:59-05:00"                                                                                                                                                                                                                 
category_id        "1370522"                                                                                                                                                                                                                                   
name               "Annual Energy Outlook 2015"                                                                                                                                                                                                                
data_set           "AEO.2015"                                                                                                                                                                                                                                  
identifier         "AEO.2015"                                                                                                                                                                                                                                  
title              "Annual Energy Outlook 2015"                                                                                                                                                                                                                
description        "The Annual Energy Outlook (AEO) from EIA.gov provides long term forecasts (25 years) of U.S. energy production, consumption, and trade for the United Stated of electricity, petroleum, natural gas, coal, nuclear, and renewable sources."
keyword            "Annual Energy Outlook AEO EIA Energy Information Administration long term forecast projection united states production consumption trade electricity petroleum natural gas coal nuclear renewable hydroelectric wind solar"                
publisher          "U.S. Energy Information Administration"                                                                                                                                                                                                    
person             "Mark Elbert"                                                                                                                                                                                                                               
mbox               "data@eia.gov"                                                                                                                                                                                                                              
accessLevel        "public"                                                                                                                                                                                                                                    
accessLevelComment "series copyright field contains further information "                                                                                                                                                                                      
accessURL          "http://api.eia.gov/bulk/AEO2015.zip"                                                                                                                                                                                                       
webService         "http://api.eia.gov"                                                                                                                                                                                                                        
format             "XML and JSON data API, JSON download file"                                                                                                                                                                                                 
temporal           "annual"                                                                                                                                                                                                                                    
modified           "2016-01-04T10:35:59-05:00"                                                                                                                                                                                                                 

HEAD_response 

Response [http://api.eia.gov/bulk/SEDS.zip]
  Date: 2020-03-16 23:57
  Status: 200
  Content-Type: application/zip
<EMPTY BODY>

# setwd("C:/Users/Christina Chung/Desktop/EO43")
# fuel <- read.csv('fuel_uncleaned.csv')
# library(dplyr)
# library(stringr) # for replacing strings
# fuel <- as.data.frame(t(fuel))
# fuel <- fuel[,2:10]
# names(fuel)<-lapply(fuel[1,],as.character)
# fuel <- fuel[-1,]
# colnames(fuel)[1] <- 'Year'
# fuel[,1] <-str_replace_all(fuel[,1],'Year','')
# colnames(fuel)<-str_replace_all(colnames(fuel),' ','_')
# for (i in c(3,6,9)){
#   fuel[,i]<-as.numeric(gsub(",", "", fuel[,i]))
#   
# }
# fuel_ordered <-fuel[nrow(fuel):1,]
# write.csv(fuel_ordered,file = 'fuel.csv')

get_EIA_bulk_json <- function(url){
  
  require(utils)
  require(here)
  require(stringr)
  require(zip)
  require(fs)
  
  os <- Sys.info()[['sysname']] 
  sep <- ifelse(os == "Windows", "\\", "/")
  
  zip_filename <- word(url, start=-1, end=-1, sep=sep)
  
  download_dir <- here("data", "raw_data", "EIA")
  path2zip     <- paste(download_dir, zip_filename, sep=sep)
  
  download.file(url = url, destfile = path2zip, method = "libcurl")
  
  unzip(path2zip, exdir = download_dir)
  
  unzipped_filename <- zip_list(path2zip)$filename
  path2unzip <- paste(download_dir,unzipped_filename,sep=sep)
  json_filename <- str_replace(unzipped_filename, ".txt", ".json")
  path2json <- paste(download_dir,json_filename,sep=sep)
  
  file_move(path2unzip, path2json)
  
  file_delete(path2zip) 
  
  return(path2json)
}

SEDS_va_elec

Error: object 'SEDS_va_elec' not found

url <- "http://api.eia.gov/bulk/EBA.zip"

path2json <- get_EIA_bulk_json(url)

trying URL 'http://api.eia.gov/bulk/EBA.zip'
Content type 'application/zip' length 284939963 bytes (271.7 MB)
==================================================
downloaded 271.7 MB

EBA <- stream_in(file(path2json))

opening file input connection.

 Found 500 records...
 Found 1000 records...
 Found 1500 records...
 Found 2000 records...
 Found 2500 records...
 Found 2789 records...
 Imported 2789 records. Simplifying...

closing file input connection.

(EBA %>%
  as_tibble %>%
  filter(geography == "USA-VA") %>%
  flatten() %>%
  mutate_all(as.character) -> EBA_va)

Error in eval(lhs, parent, parent) : object 'EBA' not found

(ELEC_va <- read_fst(path = here("data", "ELEC_va.fst")))